import os

from hdfs import InsecureClient


def upload_files_to_hdfs():
    """上传本地文件夹中的文件到HDFS"""

    # 配置参数
    url = 'http://192.168.71.128:9870'
    local_folder = "D:/boss_flask/data"
    hdfs_folder = '/zhipin'

    try:
        # 连接到HDFS - 添加超时和重试参数
        conn = InsecureClient(url, user='keri', timeout=30, root='/')
        print('✓ 成功连接到HDFS')

        # 检查HDFS连接是否正常
        try:
            conn.list('/')
            print('✓ HDFS连接验证成功')
        except Exception as e:
            print(f'✗ HDFS连接验证失败: {e}')
            return False

        # 创建HDFS目录（如果不存在）
        try:
            if not conn.status(hdfs_folder, strict=False):
                conn.makedirs(hdfs_folder, permission=755)
                print(f'✓ 创建HDFS目录: {hdfs_folder}')
            else:
                print(f'✓ HDFS目录已存在: {hdfs_folder}')
        except Exception as e:
            print(f'✗ 创建HDFS目录失败: {e}')
            return False

        # 检查本地文件夹是否存在
        if not os.path.exists(local_folder):
            print(f'✗ 本地文件夹不存在: {local_folder}')
            return False

        # 获取本地文件列表
        files = os.listdir(local_folder)
        if not files:
            print('✗ 本地文件夹中没有文件')
            return False

        print(f'找到 {len(files)} 个文件准备上传')

        # 逐个上传文件
        success_count = 0
        for file in files:
            local_file_path = os.path.join(local_folder, file)
            hdfs_file_path = f"{hdfs_folder}/{file}"

            # 检查是否是文件（不是目录）
            if os.path.isfile(local_file_path):
                try:
                    print(f'正在上传: {file}')

                    # 使用正确的上传方法
                    with open(local_file_path, 'rb') as local_file:
                        conn.write(hdfs_file_path, local_file.read(), overwrite=True)

                    success_count += 1
                    print(f'✓ 上传成功: {file}')

                except Exception as e:
                    print(f'✗ 上传失败 {file}: {e}')
            else:
                print(f'跳过目录: {file}')

        print(f'\n✓ 上传完成: {success_count}/{len(files)} 个文件成功')
        return success_count > 0

    except Exception as e:
        print(f'✗ 连接失败: {e}')
        return False


# 运行上传函数
if __name__ == "__main__":
    upload_files_to_hdfs()